# -*- coding: utf-8 -*-
"""
Created on Wed Nov  9 13:38:46 2022

@author: sthgy
"""


import os
import time
import datetime
import requests
import numpy as np
import pandas as pd
import akshare as ak
from random import uniform
from bs4 import BeautifulSoup

def date_format(date_string):
    """ 清洗日期格式 """
    if '---' in date_string:
        return pd.NaT
    elif '-' in date_string:
        return datetime.datetime.strptime(date_string, '%Y-%m-%d').date()
    else:
        raise ValueError(f"日期数据格式认不得咯：'{date_string}'")

def numeric_format(numeric_string):
    """ 清洗数值格式 """
    if '-' in numeric_string:
        return np.nan
    elif numeric_string[-1] == '*':
        return(float(numeric_string[:-1]))
    else:
        return float(numeric_string)

def check_dir():
    ''' 检查数据文件路径是否健全 '''
    if not os.path.exists('./FundData/'):
        os.makedirs('./FundData/')

def print_progress(text, finished_num, total_num):
    """ 打印进度用 """
    print(f'\r{text}: {finished_num} / {total_num}', end='')

def download_fund_info():
    ''' 下载当下基金基本信息 '''
    fund_info = ak.fund_name_em()
    with pd.HDFStore('./FundData/data.h5') as hdf_file:
        fund_info.to_hdf(hdf_file, 'fund_info', complevel=3)

def read_sift_fund():
    """ 读取并筛选全部基金
    剔除了货币、商品、REITs、FOF等类型"""
    with pd.HDFStore('./FundData/data.h5') as hdf_file:
        fund_info = pd.read_hdf(hdf_file, 'fund_info')
    fund_sift = fund_info.loc[fund_info['基金类型'].apply(lambda x: x in [
        '混合型-灵活',
        '债券型-可转债',
        '债券型-长债',
        '混合型-偏股',
        '指数型-股票',
        '债券型-混合债',
        '股票型',
        '债券型-中短债',
        '混合型-偏债',
        '混合-绝对收益',
        '混合型-平衡'
        ]), :]
    return fund_sift

# akshare没有获取历史数据，需要从天天基金爬
def download_fund_scale():
    ''' 基金规模数据 '''
    fund_info = read_sift_fund()
    df_dict = {}
    count = 1
    total = len(fund_info['基金代码'])
    for fund_code in fund_info['基金代码']:
        try:
            print_progress(f'获取基金{fund_code}规模数据', count, total)
            url = f'http://fundf10.eastmoney.com/FundArchivesDatas.aspx?type=gmbd&mode=0&code={fund_code}'
            text = requests.get(url).text
            if '<table' in text:
                text_clean = text[text.find('<table'):text.find('</table>')+len('</table>')]
                if '<thead' in text_clean and '<tbody' in text_clean:
                    soup = BeautifulSoup(text_clean, 'html.parser')
                    col_name = [x.text for x in soup.thead.find_all('th')]
                    data = [
                        [y.text for y in x.find_all('td')]
                        for x in soup.tbody.find_all('tr')]
                    df = pd.DataFrame(data, columns=col_name)
                    df_dict[fund_code] = df
            time.sleep(uniform(1,3))
        except Exception as e:
            print(f'{fund_code} 出现异常', e)
        finally:
            count += 1
    df_list = []
    for fund_code, df in df_dict.items():
        # 最后一行可能存在数据说明
        df = df.loc[df['日期'].apply(lambda x: '说明' not in x and '*' not in x), :]
        df_clean = pd.DataFrame()
        df_clean['基金代码'] = np.repeat(fund_code, df.shape[0])
        df_clean['日期'] = df['日期'].apply(date_format)
        df_clean['期末净资产（亿元）'] = df['期末净资产（亿元）'].apply(numeric_format)
        df_list.append(df_clean)
    df_all = pd.concat(df_list, ignore_index=True)
    with pd.HDFStore('./FundData/data.h5') as hdf_file:
        df_all.to_hdf(hdf_file, 'fund_scale')

# 基金业绩
def download_fund_cumnav():
    ''' 基金净值数据 '''
    # 累计收益率数据不全，累计净值剔除了分红的影响
    # 定开基金只有周频净值，以及有出现周频和日频混合的情况
    fund_info = read_sift_fund()
    df_dict = {}
    count = 1
    total = len(fund_info['基金代码'])
    for fund_code in fund_info['基金代码']:
        try:
            print_progress(f'获取基金{fund_code}累计净值数据', count, total)
            df = ak.fund_open_fund_info_em(fund=fund_code, indicator="累计净值走势")
            df_dict[fund_code] = df
            time.sleep(uniform(1,3))
        except Exception as e:
            print(f'{fund_code} 出现异常', e)
        finally:
            count += 1
    df_list = []
    for fund_code, df in df_dict.items():
        # 非A份额没有数据
        if df.shape[0] > 0:
            df_clean = pd.DataFrame()
            df_clean['基金代码'] = np.repeat(fund_code, df.shape[0])
            df_clean['日期'] = df['净值日期']
            df_clean['累计净值'] = df['累计净值']
            df_list.append(df_clean)
    df_all = pd.concat(df_list, ignore_index=True)
    with pd.HDFStore('./FundData/data.h5') as hdf_file:
        df_all.to_hdf(hdf_file, 'fund_cumnav')

if __name__ == '__main__':
    check_dir()
    download_fund_info()
    fund_info = read_sift_fund()
    download_fund_scale() # 预计运行10个小时左右
    download_fund_cumnav() # 预计运行10个小时左右